package org.huangrui.spark.scala.core.rdd.operate.transform

import org.apache.spark.{SparkConf, SparkContext}

/**
 * @Author hr
 * @Create 2024-10-17 8:27 
 */
object Spark06_Operate_Transform_Distinct {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").setAppName("spark")
    val sc = new SparkContext(conf)
    val nums = List(1, 2, 3, 2, 3, 2, 1, 2, 3, 1)
    val rdd = sc.parallelize(nums, 3)
    // TODO distinct : 去重
    //      hashSet去重，是单点去重
    //      distinct ，是分布式去重, 采用了分组+shuffle的处理方式
    rdd.distinct().foreach(println)

    sc.stop()
  }
}
